package com;

import org.apache.spark.SparkConf;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

import java.util.ArrayList;
import java.util.List;

public class Sixoo {

    public static void main(String[] args) {

        SparkConf conf = new SparkConf();
        SparkSession spark = SparkSession
                .builder()
                .appName("Java Spark SQL basic example")
                .config("spark.io.compression.codec", "snappy")
                .config(conf)
                .master("local[4]")
                .getOrCreate();


        List<StructField> fields = new ArrayList<>();
        fields.add(DataTypes.createStructField("channel", DataTypes.StringType, true));
        fields.add(DataTypes.createStructField("time", DataTypes.LongType, true));
        for (int i = 1; i <= 3000; i++) {
            fields.add(DataTypes.createStructField("p" + i, DataTypes.IntegerType, true));
        }
        for (int i = 3001; i <= 15000; i++) {
            fields.add(DataTypes.createStructField("p" + i, DataTypes.ShortType, true));
        }
        StructType schema = DataTypes.createStructType(fields);
        System.out.println(schema.size());

        Dataset<Row> data = spark.read()
                .option("header", "false")
                .csv("/home/hdfs/soft/IdeaProjects/dec-kks-etl/fly1.txt");

        data.show();
//        spark.createDataFrame(list, schema).show();


    }
}
